#
# -*- coding: utf-8 -*-

'''
思路：
1. 获取所有食品类的产品的SKU_ID，即http://item.jd.com/1124783.html中的1124783
2. 通过SKU_ID获取该产品的包装图，即http://item.jd.com/bigimage.aspx?id=SKU_ID访问得到
3. ...
'''

import urllib
import json
import re

#lxml专门分析网页的，我们主要使用etree和他的xpath方法
from lxml import etree

import sys
reload(sys)
sys.setdefaultencoding("utf-8")

config_file = "JD.csv"

#分类地址： 进口食品\饼干蛋糕 http://list.jd.com/list.html?cat=1320,5019,5020
#全部分类地址：http://www.jd.com/allSort.aspx

class JDParser(object):

    def __init__(self, url):
            self.url = url
            self._response = urllib.urlopen(self.url)
            self.html = self._response.read()

    #分析该分类的总页数
    def get_pages(self):
        '''<div class="f-pager" id="J_topPage">
           <span class="fp-text">
               <b>1</b><em>/</em><i>205</i>
           </span>
        '''
        tree = etree.HTML(self.html)
        #通过xpath定位到我们需要的tag处，即找到 “共204页” 处
        con = tree.xpath("/html/body/div[8]/div[1]/div[2]/div[1]/div/div[1]/div[1]/div[2]/span/i")
        pages = 0
        #获取该分类的总商品数
        for l in con:
            #print l.text
            pages = int(l.text)
        #print con.text
        return pages

    #分析SKU_ID
    def get_content(self):
        global filename
        tree = etree.HTML(self.html)
        con = tree.xpath("/html/body/div[8]/div[1]/div[2]/div[1]/div/div[3]/ul")
        for li in con:
            for l in li:
                #print type(l[0][2][0]), l[0][2][0].attrib["href"], l[0][2][0][0].text
                #print l[0][2][0].attrib["href"], l[0][2][0][0].text
                p_url = l[0][2][0].attrib["href"]
                p_name = l[0][2][0][0].text
                # p_url 包含了sku_id 信息
                p_sku=p_url[14:len(p_url)-5]
                filename.write(p_sku)
                filename.write(',')
                filename.write(p_url)
                filename.write(',')
                filename.write(p_name)
                filename.write('\n')
    '''
    def get_pic(self):
        # http://item.jd.com/bigimage.aspx?id=2625329
        tmp = self.html
        nPos = tmp.index("newOutputAllImages.data")
        nPos2 = tmp.index("}];")
        #nPos2 = tmp.index("newOutputAllImages.init")
        ttt = tmp[nPos+24:nPos2+2]
        #print ttt
        #print "=================================================================="
        dic = json.loads(ttt)
        #print dic
        print "pics:", len(dic)
        for p in dic:
            #将URL中的n5替换为n0即为大图
            #old //img14.360buyimg.com/n5/jfs/t292/138/1025254446/203494/e0e63c96/542d0e2bNacf96e70.jpg
            #new //img14.360buyimg.com/n0/jfs/t292/138/1025254446/203494/e0e63c96/542d0e2bNacf96e70.jpg
            old = p['img']
            new = old.replace("com/n5/jfs", "com/n0/jfs", 1)
            #print new
    '''            

class classification(object):
    def __init__(self, url):
        self.url = url
        self.pages = 0
    
    def getPages(self):
        self.pages = JDParser(self.url).get_pages()
    
    def run(self):
        self.getPages() #多构造一次JDParser对象
        for i in range(self.pages):
            #http://list.jd.com/list.html?cat=1320,5019,5020&page=2&trans=1&JL=6_0_0
            print 'pages: '+str(i+1)+'/'+str(self.pages)
            newUrl = self.url + "&page=" + str(i+1) + "&trans=1&JL=6_0_0"
            nJP = JDParser(newUrl)
            nJP.get_content()
            

class PictureParser(object):
    def __init__(self, sku):
        self.sku = sku
        self._response = urllib.urlopen("http://item.jd.com/bigimage.aspx?id="+self.sku)
        self.html = self._response.read()
        
    #商品预览图地址
    #http://item.jd.com/bigimage.aspx?id=1217524   sku_id
    def get_pic(self):
        tmp = self.html
        nPos = tmp.index("newOutputAllImages.data")
        nPos2 = tmp.index("}];")
        #nPos2 = tmp.index("newOutputAllImages.init")
        ttt = tmp[nPos+24:nPos2+2]
        #print ttt
        #print "=================================================================="
        dic = json.loads(ttt)
        #print dic
        print "pics:", len(dic)
        count = 0
        for p in dic:
            #old //img14.360buyimg.com/n5/jfs/t292/138/1025254446/203494/e0e63c96/542d0e2bNacf96e70.jpg
            #new //img14.360buyimg.com/n0/jfs/t292/138/1025254446/203494/e0e63c96/542d0e2bNacf96e70.jpg
            old = p['img']
            new = old.replace("com/n5/", "com/n0/", 1)
            #print new
            count = count+1
            print str(count)+"/"+str(len(dic))
            #http://www.pythontab.com/html/2014/pythonjichu_0121/681.html
            #默认写到当前目录的images子目录
            urllib.urlretrieve("http:"+new, "images\\"+self.sku+"_"+str(count)+".jpg")


class PictureDownloader(object):
    def __init__(self, filename):
        self.filename = filename

    def run(self):
        f = open(self.filename, 'r')
        count = 0
        for line in f.readlines():
            count = count + 1
            dic = line.split(',')
            #print dic
            print "download",dic[0]
            newPP = PictureParser(dic[0])
            newPP.get_pic()

        
if __name__ == '__main__':
        
        global filename
	filename = open(config_file, 'a')
	#进口食品
	#c = classification('http://list.jd.com/list.html?cat=1320,5019,5020')
	#月饼
	c = classification('https://list.jd.com/list.html?cat=1320,2641,2642')
	c.run()
	filename.close()
	
	
	nJP = PictureDownloader(config_file)
	nJP.run()
	

'''
#京东分类
<dl class="clearfix">
<dt><a href="http://channel.jd.com/1320-1581.html" target="_blank">地方特产</a></dt>
<dd>
<a href="http://list.jd.com/list.html?cat=1320,1581,12217" target="_blank">其他特产</a>
<a href="http://list.jd.com/list.html?cat=1320,1581,1589" target="_blank">新疆</a>
<a href="http://list.jd.com/list.html?cat=1320,1581,2644" target="_blank">北京</a>
<a href="http://list.jd.com/list.html?cat=1320,1581,2647" target="_blank">山西</a>
<a href="http://list.jd.com/list.html?cat=1320,1581,2648" target="_blank">内蒙古</a>
<a href="http://list.jd.com/list.html?cat=1320,1581,2653" target="_blank">福建</a>
<a href="http://list.jd.com/list.html?cat=1320,1581,2656" target="_blank">湖南</a>
<a href="http://list.jd.com/list.html?cat=1320,1581,2669" target="_blank">四川</a>
<a href="http://list.jd.com/list.html?cat=1320,1581,2670" target="_blank">云南</a>
<a href="http://list.jd.com/list.html?cat=1320,1581,4693" target="_blank">东北</a>
</dd>
</dl>
<dl class="clearfix">
<dt><a href="http://channel.jd.com/1320-1583.html" target="_blank">休闲食品</a></dt>
<dd>
<a href="http://list.jd.com/list.html?cat=1320,1583,1590" target="_blank">休闲零食</a>
<a href="http://list.jd.com/list.html?cat=1320,1583,1591" target="_blank">坚果炒货</a>
<a href="http://list.jd.com/list.html?cat=1320,1583,1592" target="_blank">肉干肉脯</a>
<a href="http://list.jd.com/list.html?cat=1320,1583,1593" target="_blank">蜜饯果干</a>
<a href="http://list.jd.com/list.html?cat=1320,1583,1594" target="_blank">糖果/巧克力</a>
<a href="http://list.jd.com/list.html?cat=1320,1583,1595" target="_blank">饼干蛋糕</a>
<a href="http://list.jd.com/list.html?cat=1320,1583,7121" target="_blank">无糖食品</a>
</dd>
</dl>
<dl class="clearfix">
<dt><a href="http://channel.jd.com/1320-1584.html" target="_blank">粮油调味</a></dt>
<dd>
<a href="http://list.jd.com/list.html?cat=1320,1584,2675" target="_blank">米面杂粮</a>
<a href="http://list.jd.com/list.html?cat=1320,1584,2676" target="_blank">食用油</a>
<a href="http://list.jd.com/list.html?cat=1320,1584,2677" target="_blank">调味品</a>
<a href="http://list.jd.com/list.html?cat=1320,1584,2678" target="_blank">南北干货</a>
<a href="http://list.jd.com/list.html?cat=1320,1584,2679" target="_blank">方便食品</a>
<a href="http://list.jd.com/list.html?cat=1320,1584,2680" target="_blank">有机食品</a>
</dd>
</dl>
<dl class="clearfix">
<dt><a href="http://channel.jd.com/1320-1585.html" target="_blank">饮料冲调</a></dt>
<dd>
<a href="http://list.jd.com/list.html?cat=1320,1585,10975" target="_blank">饮用水</a>
<a href="http://list.jd.com/list.html?cat=1320,1585,1602" target="_blank">饮料</a>
<a href="http://list.jd.com/list.html?cat=1320,1585,9434" target="_blank">牛奶乳品</a>
<a href="http://list.jd.com/list.html?cat=1320,1585,3986" target="_blank">咖啡/奶茶</a>
<a href="http://list.jd.com/list.html?cat=1320,1585,1601" target="_blank">冲饮谷物</a>
<a href="http://list.jd.com/list.html?cat=1320,1585,12200" target="_blank">蜂蜜/柚子茶</a>
<a href="http://list.jd.com/list.html?cat=1320,1585,12201" target="_blank">成人奶粉</a>
</dd>
</dl>
'''
